import random
import math
import datetime
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import Core.DataSeries as DataSeries
import Core.Gadget as Gadget
from sklearn.model_selection import train_test_split, learning_curve, validation_curve, ShuffleSplit, GridSearchCV
from sklearn.tree import DecisionTreeRegressor
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import make_scorer
from sklearn import metrics
import Factors

#
def RangeReturn_Dic(database, datetime1, datetime2):

    # Mid-Night
    # datetime2 = datetime2 + datetime.timedelta(days=1)
    filter = {}
    filter["DateTime"] = {">=": datetime1, "<=": datetime2}
    filter["$and"] = [{"Date":{">=": datetime1.date()}}, {"Date":{"<=": datetime2.date()}}]
    # filter["limit"] = 1000
    data = database.Find("DailyBar", "Stock", filter,
                         projection=["DateTime","Symbol","BClose"],
                         sort=[("DateTime",1)])

    # Loop and Storage
    barsBySymbols = {}
    for d in data:
        symbol = d["Symbol"]
        bClose = d["BClose"]
        dt = d["DateTime"]
        if symbol not in barsBySymbols.keys():
            barsBySymbols[symbol] = []
        #
        barsBySymbols[symbol].append({"DateTime": dt, "BClose": bClose})

    #
    returnBySymbols = {}
    for symbol, bars in barsBySymbols.items():
        if symbol == "000001.SZ":
            a = bars[-1]["BClose"]
            b = bars[0]["BClose"]
            pass
        returnBySymbols[symbol] = bars[-1]["BClose"] / bars[0]["BClose"] - 1

    #
    return returnBySymbols


def RangeReturn_Dic_Fast(database, datetime1, datetime2):
    pass


#
def RangeReturn(database, datetime1, datetime2, instruments=None, benchmarkSymbol=None, dfSymbols=None):
    # print("RengeReturn, Loading from Database, May Slowly")
    # ---Load Database---
    returnBySymbols = RangeReturn_Dic(database, datetime1, datetime2)

    # ---Build Return Table---
    data = []

    symbols = []
    if instruments != None:
        for instrument in instruments:
            symbol = instrument["Symbol"]
            symbols.append(symbol)
    if dfSymbols != None:
        symbols = dfSymbols["Symbol"].tolist()
    else:
        symbols = returnBySymbols.keys()

    for symbol in symbols:
        if symbol not in returnBySymbols:
            print("NO Return", symbol)
            continue
        ret = returnBySymbols[symbol]
        data.append([symbol, ret])
    #
    df = pd.DataFrame(data, columns=["Symbol", "Return"])

    # ---Excess Return---
    if benchmarkSymbol == None: # 不指定Symbol就用平均值为BM
        benchmarkReturn = df["Return"].mean()
    else:
        # Mid-Night
        datetime2 = datetime2 + datetime.timedelta(days=1)
        filter = {}
        filter["DateTime"] = {">=": datetime1, "<=": datetime2}
        data = database.Find("Stock", "DailyBar", filter, sort=[("DateTime", 1)])
        benchmarkReturn = data[-1]["BClose"] / data[0]["BClose"] - 1

    # Excess Return
    df["BenchmarkReturn"] = benchmarkReturn
    df["ExcessReturn"] = df["Return"] - benchmarkReturn

    #
    return df


#
def FactorDataSeries(database, symbol, factorName):

    #
    # factorName = "Leverage"
    filter = {"Symbol": symbol}
    #
    dataSeries = database.Find("Factor", factorName, filter, sort=[("DateTime", 1)])

    df = pd.DataFrame()

    data = []
    for value in dataSeries:
        data.append([value["ReportDate"], value["DateTime"], value["Value"]])

    #
    df = pd.DataFrame(data, columns=["ReportDate", "DateTime", symbol])
    return df


def PeriodReturn_DataFrame(database, symbol, datetime1, datetime2, instrumentType="Stock",
                            weekly=False, alignCanlenderDays=False):
    #
    dfTrandingDays = Gadget.GetTradingDays_DataFrame(database, datetime1, datetime2, backforword1Day=True)
    datetime1 = dfTrandingDays.iloc[0]["Date"]
    datetime2 = dfTrandingDays.iloc[-1]["Date"]
    #
    filter = {}
    filter["Date"] = {">=": datetime1, "<=": datetime2}
    filter["Symbol"] = symbol
    #
    fields = "BClose"
    if "index" in instrumentType.lower():
        fields = "Close"
    bmRets = database.Find("DailyBar", instrumentType, filter)
    #
    df = Gadget.DocumentsToDataFrame(bmRets, keep=["Date", fields])
    df.rename(columns={fields: "Close"}, inplace=True)
    # df["Date"] = pd.to_datetime(df["Date"])
    # print(df.columns)
    #
    if weekly:
        #
        if alignCanlenderDays:
            dfCalender = Gadget.GenerateCalenderDays(datetime1, datetime2)
            df = pd.merge(dfCalender, df, how="left", on="Date")
            df.fillna(method="ffill", inplace=True)
            #
        Gadget.AddWeekColumn(df)
        df = df.groupby('Week').nth(-1)
        df["Return"] = df["Close"] / df["Close"].shift(1) - 1
    else:
        df["Return"] = df["Close"] / df["Close"].shift(1) - 1
        df = df.iloc[1:] # 抛弃第一格

    #
    # print(df)
    df.rename(columns={"Return": symbol}, inplace=True)
    return df[["Date", symbol]]

#

def Profile2(database, factors, reportDate, datetimeField="ReportDate"):
    #
    df = pd.DataFrame()
    #
    filter = {}
    filter[datetimeField] = reportDate
    for factor in factors:
        #
        favtorValues = database.Find("Factor", factor, filter)
        data = []
        for value in favtorValues:
            data.append([value["Symbol"], value["Value"]])

            if value["Symbol"] == "600562.SZ":
                a = 1

        dfTemp = pd.DataFrame(data, columns=["Symbol", factor])
        #
        if df.empty:
            df = dfTemp
        else:
            df = pd.merge(df, dfTemp, how="outer", on="Symbol")
    #
    return df


def Profile(database, datetime2, factorList, instruments=None):

    #
    def MostDateTime(documents, datetimeField):
        dic = {}
        for doc in documents:
            dt = doc[datetimeField]
            if dt not in dic.keys():
                dic[dt] = 0
            dic[dt] += 1
        #
        sortedList = sorted(dic.items(), key=lambda item: item[1])
        # return last item(max), first value(DateTime)
        return sortedList[-1][0]

    #
    datetime0 = datetime2 + datetime.timedelta(days=-365)

    print(" ### Profile Data", datetime2, "###")

    factorParams = Factors.FactorParams()

    # ---Loop Factor to Load---
    df = pd.DataFrame()
    for factor in factorList:
        #
        print("Try to Load", factor)

        # ---校准时间---
        # tempDateTime1 = datetime.datetime(2018,1,1)
        # tempDateTime2 = datetime.datetime(2019,1,1)
        #
        # filter = {}
        # filter["$and"] = [{"DateTime": {">=": tempDateTime1}}, {"DateTime": {"<=": tempDateTime2}}]
        # filter["$and"].append({"Symbol":"000001.SZ"})
        # dfFiguerTime = database.Find("Factor", factor, filter)
        # count = len(dfFiguerTime)
        #
        # dfBase = FactorDataSeries(database, "000001.SZ", factor)
        # dfBase = dfBase[dfBase["DateTime"] <= datetime2]
        # #
        # if dfBase.shape[0] == 0:
        #     print("Factor No Value", factor, datetime2)
        #     continue
        #
        # dfBase = dfBase.iloc[-1]
        # # print(dfBase)

        # ---校准时间 新方法---
        filter = {}
        filter["limit"] = 1000
        filter["DateTime"] = {"<=": datetime2, ">": datetime0}
        sort = [("DateTime", -1)]
        tempDocs = database.Find("Factor", factor, filter, sort)
        #
        if len(tempDocs) == 0:
            print("Factor No Value", factor, datetime2)
            continue
        #
        tempDoc = tempDocs[0]
        #
        param = factorParams.get(factor)
        period = param.get("Period")

        #
        if period == "Quarterly" or period == "ReportDate":# Fundamental Factor
            datetimeField = "ReportDate"
        else:
            datetimeField = "DateTime"

        #
        targetDatetime = MostDateTime(tempDocs, datetimeField)

        # ---Load Factors on Specific Date---
        dfTemp = Profile2(database, [factor], targetDatetime, datetimeField)
        dfTemp2 = dfTemp.dropna()
        print("Loaded#", dfTemp.shape[0], "#Null", dfTemp.shape[0]-dfTemp2.shape[0], "@", datetimeField, targetDatetime)
        if df.empty:
            df = dfTemp
        else:
            df = pd.merge(df, dfTemp, how="outer", on="Symbol")

    #
    if instruments != None:
        data = []
        for instrument in instruments:
            data.append(instrument["Symbol"])
            dfSymbols = pd.DataFrame(data, columns=["Symbol"])
        #
        df = pd.merge(dfSymbols, df, how="left", on="Symbol")

    #
    return df


def Profile_MostRecent(database, datetime2, factors, instruments=None):
    #
    df = pd.DataFrame()

    #
    # 不设起点，只设终点，可能数据量会很大
    filter = {}
    datetime0 = datetime2 + datetime.timedelta(days=-365) # 必要时加入起点
    filter["DateTime"] = {"<=": datetime2}
    for factor in factors:
        #
        factorValuesBySymbol = {}
        #
        factorValues = database.Find("Factor", factor, filter)
        for factorValue in factorValues:
            symbol = factorValue["Symbol"]
            value = factorValue["Value"]
            dt = factorValue["DateTime"]
            #
            if symbol not in factorValuesBySymbol.keys():
                factorValuesBySymbol[symbol] = []
            #
            factorValuesBySymbol[symbol].append([value, dt])
        #
        data = []
        for symbol, values in factorValuesBySymbol.items():
            data.append([symbol, values[-1][0]])# 0-Value, 1-DateTime
        #
        dfTemp = pd.DataFrame(data, columns=["Symbol", factor])
        #
        if df.empty:
            df = dfTemp
        else:
            df = pd.merge(df, dfTemp, how="left", on="Symbol")

    #
    if instruments != None:
        data = []
        for instrument in instruments:
            data.append(instrument["Symbol"])
            dfSymbols = pd.DataFrame(data, columns=["Symbol"])
        #
        df = pd.merge(dfSymbols, df, how="left", on="Symbol")

    #
    return df


def Profile_to_File(database, datetime2, factors, instruments=None, folderName=""):
    #
    df = Profile(database, datetime2, factors, instruments)
    df["DateTime"] = datetime2.date()

    # print(df.head())

    if folderName == None or folderName == "":
        return

    folderName = "d://Data//Profile//" + folderName + "//"
    Gadget.CreateFolder(folderName)
    # print(df)
    pathFilename = folderName + "Profile_" + Gadget.ToDateString(datetime2) + ".csv"
    print("Export Profile Data", pathFilename)
    df.to_csv(pathFilename)


def RangeReturn_to_File(database, datetime1, datetime2, instruments, benchmarkSymbol=None, folderName=""):
    #
    df = RangeReturn(database, datetime1, datetime2, instruments, benchmarkSymbol)
    folderName = "d://Data//RangeReturn//" + folderName + "//"
    Gadget.CreateFolder(folderName)
    df.to_csv(folderName + "RangeReturn_" + Gadget.ToDateString(datetime1) + "_" + Gadget.ToDateString(datetime2) + ".csv")


def ExportDataFiles(database,
                    datetime1, datetime2,
                    factors, instruments,
                    profileFolderName=None, rangeReturnFolderName=None):
    #
    if profileFolderName != None:
        Profile_to_File(database, datetime1, factors, instruments=instruments, folderName=profileFolderName)
    #
    if rangeReturnFolderName != None:
        RangeReturn_to_File(database, datetime1, datetime2, instruments, benchmarkSymbol=None, folderName=rangeReturnFolderName)


def ExportDataFilesMultiPeriod(database,
                               datetime1, datetime2,
                               factors, instruments,
                               profileFolderName=None, rangeReturnFolderName=None):
    #
    datatimeRanges = Gadget.GenerateTimeRange_Yearly(datetime1, datetime2)
    # datatimeRanges = Gadget.GenerateTimeRange_HalfYear(datetime1, datetime2)

    #
    for timeRange in datatimeRanges:
        begin = timeRange[0]
        end = timeRange[1]
        ExportDataFiles(database, begin, end, factors, instruments, profileFolderName, rangeReturnFolderName)


def PerformanceTest_WinRatio(database, df, datetime1, datetime2, dfReturn=pd.DataFrame(), comment=""):
    print("")
    print("PerformanceTest WinRatio", Gadget.ToDateString(datetime1), Gadget.ToDateString(datetime2))
    # ---Total Market(Total Sample)---
    if dfReturn.empty:
        dfReturn = RangeReturn(database, datetime1, datetime2)
    # print(dfReturn.head(10))
    bmReturnMed = dfReturn["Return"].median()
    bmReturnAvg = dfReturn["Return"].mean()
    totalCount = len(dfReturn)

    # ---Strategy Selected---
    dfRaw = pd.merge(df, dfReturn, how="left", on="Symbol")
    dfRaw.to_csv("d://data//Performance//" + comment + "_" + Gadget.ToDateString(datetime1) + "_" + Gadget.ToDateString(
        datetime2) + ".csv")

    #
    winCount = dfRaw[dfRaw["Return"] > bmReturnMed].shape[0]
    selectReturnMed = dfRaw["Return"].median()
    selectReturnAvg = dfRaw["Return"].mean()
    selectCount = len(dfRaw)
    print(comment, "TotalCount", totalCount, "Med", "%.4f"%(bmReturnMed), "Avg", "%.4f"%(bmReturnAvg))
    print(comment, "SeletCount", selectCount, "Med", "%.4f"%(selectReturnMed), "Avg", "%.4f"%(selectReturnAvg), "WinRatio", "%.4f"%(winCount / selectCount))

    #
    return dfReturn


def PerformanceTest_InformationCoefficient(database, df, datetime1, datetime2, ranking=True, dfReturn=pd.DataFrame()):
    # ---Redundant---
    numeric_features = list(df.columns)
    redundantFields = ["Symbol", "DateTime"]
    for field in redundantFields:
        if field in numeric_features:
            numeric_features.remove(field)

    # ---Process Next Period Return---
    if dfReturn.empty:
        dfReturn = RangeReturn(database, datetime1, datetime2)
    # print(dfReturn.head(10))
    dfRaw = pd.merge(df, dfReturn, how="left", on="Symbol")
    #dfRaw.to_csv("d://data//ProfileModel4//XY_Raw_wReturn" + Gadget.ToDateString(datetime1) + "_" + Gadget.ToDateString(
    #    datetime2) + ".csv")
    #
    if ranking:
        dfRaw["Return"] = dfRaw["Return"].rank(ascending=True, pct=True)
    for field in numeric_features:
        if ranking:
            dfRaw[field] = dfRaw[field].rank(ascending=True, pct=True)
        #
        corr = dfRaw["Return"].corr(dfRaw[field], method="pearson")
        print("Corr", datetime1, datetime2, field, corr)
    #
    return {"Corr": corr, "dfReturn": dfReturn}


if __name__ == '__main__':
    #
    from Core.Config import *
    cfgPathFilename = os.getcwd() + "/../config.json"
    config = Config(cfgPathFilename)
    database = config.DataBase("MySQL")

    #
    datetime1 = datetime.datetime(2019, 12, 1)
    datetime2 = datetime.datetime(2019, 12, 20)

    #
    RangesReturn_DataFrame2(database, "000001.SH", datetime1, datetime2, instrumentType="Index2", weekly=True)

    # ---Instruments---
    # instruments = database.Find("Instruments", "Stock")

    # ---Export Files---
    # ExportDataFilesMultiPeriod(database, datetime1, datetime2, None, instruments,
    #                         profileFolderName=None, rangeReturnFolderName="Temp")